tcg/tcg-op-vec.c

   1 /*
   2  * Tiny Code Generator for QEMU
   3  *
   4  * Copyright (c) 2018 Linaro, Inc.
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "qemu/osdep.h"
  21 #include "cpu.h"
  22 #include "tcg/tcg.h"
  23 #include "tcg/tcg-op.h"
  24 #include "tcg/tcg-mo.h"
  25
  26 /* Reduce the number of ifdefs below.  This assumes that all uses of
  27    TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
  28    the compiler can eliminate.  */
  29 #if TCG_TARGET_REG_BITS == 64
  30 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
  31 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
  32 #define TCGV_LOW  TCGV_LOW_link_error
  33 #define TCGV_HIGH TCGV_HIGH_link_error
  34 #endif
  35
  36 /*
  37  * Vector optional opcode tracking.
  38  * Except for the basic logical operations (and, or, xor), and
  39  * data movement (mov, ld, st, dupi), many vector opcodes are
  40  * optional and may not be supported on the host.  Thank Intel
  41  * for the irregularity in their instruction set.
  42  *
  43  * The gvec expanders allow custom vector operations to be composed,
  44  * generally via the .fniv callback in the GVecGen* structures.  At
  45  * the same time, in deciding whether to use this hook we need to
  46  * know if the host supports the required operations.  This is
  47  * presented as an array of opcodes, terminated by 0.  Each opcode
  48  * is assumed to be expanded with the given VECE.
  49  *
  50  * For debugging, we want to validate this array.  Therefore, when
  51  * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders
  52  * will validate that their opcode is present in the list.
  53  */
  54 #ifdef CONFIG_DEBUG_TCG
  55 void tcg_assert_listed_vecop(TCGOpcode op)
  56 {
  57     const TCGOpcode *p = tcg_ctx->vecop_list;
  58     if (p) {
  59         for (; *p; ++p) {
  60             if (*p == op) {
  61                 return;
  62             }
  63         }
  64         g_assert_not_reached();
  65     }
  66 }
  67 #endif
  68
  69 bool tcg_can_emit_vecop_list(const TCGOpcode *list,
  70                              TCGType type, unsigned vece)
  71 {
  72     if (list == NULL) {
  73         return true;
  74     }
  75
  76     for (; *list; ++list) {
  77         TCGOpcode opc = *list;
  78
  79 #ifdef CONFIG_DEBUG_TCG
  80         switch (opc) {
  81         case INDEX_op_and_vec:
  82         case INDEX_op_or_vec:
  83         case INDEX_op_xor_vec:
  84         case INDEX_op_mov_vec:
  85         case INDEX_op_dup_vec:
  86         case INDEX_op_dup2_vec:
  87         case INDEX_op_ld_vec:
  88         case INDEX_op_st_vec:
  89         case INDEX_op_bitsel_vec:
  90             /* These opcodes are mandatory and should not be listed.  */
  91             g_assert_not_reached();
  92         case INDEX_op_not_vec:
  93             /* These opcodes have generic expansions using the above.  */
  94             g_assert_not_reached();
  95         default:
  96             break;
  97         }
  98 #endif
  99
 100         if (tcg_can_emit_vec_op(opc, type, vece)) {
 101             continue;
 102         }
 103
 104         /*
 105          * The opcode list is created by front ends based on what they
 106          * actually invoke.  We must mirror the logic in the routines
 107          * below for generic expansions using other opcodes.
 108          */
 109         switch (opc) {
 110         case INDEX_op_neg_vec:
 111             if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)) {
 112                 continue;
 113             }
 114             break;
 115         case INDEX_op_abs_vec:
 116             if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)
 117                 && (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0
 118                     || tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0
 119                     || tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece))) {
 120                 continue;
 121             }
 122             break;
 123         case INDEX_op_cmpsel_vec:
 124         case INDEX_op_smin_vec:
 125         case INDEX_op_smax_vec:
 126         case INDEX_op_umin_vec:
 127         case INDEX_op_umax_vec:
 128             if (tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
 129                 continue;
 130             }
 131             break;
 132         default:
 133             break;
 134         }
 135         return false;
 136     }
 137     return true;
 138 }
 139
 140 void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a)
 141 {
 142     TCGOp *op = tcg_emit_op(opc);
 143     TCGOP_VECL(op) = type - TCG_TYPE_V64;
 144     TCGOP_VECE(op) = vece;
 145     op->args[0] = r;
 146     op->args[1] = a;
 147 }
 148
 149 void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece,
 150                TCGArg r, TCGArg a, TCGArg b)
 151 {
 152     TCGOp *op = tcg_emit_op(opc);
 153     TCGOP_VECL(op) = type - TCG_TYPE_V64;
 154     TCGOP_VECE(op) = vece;
 155     op->args[0] = r;
 156     op->args[1] = a;
 157     op->args[2] = b;
 158 }
 159
 160 void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece,
 161                TCGArg r, TCGArg a, TCGArg b, TCGArg c)
 162 {
 163     TCGOp *op = tcg_emit_op(opc);
 164     TCGOP_VECL(op) = type - TCG_TYPE_V64;
 165     TCGOP_VECE(op) = vece;
 166     op->args[0] = r;
 167     op->args[1] = a;
 168     op->args[2] = b;
 169     op->args[3] = c;
 170 }
 171
 172 static void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r,
 173                       TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e)
 174 {
 175     TCGOp *op = tcg_emit_op(opc);
 176     TCGOP_VECL(op) = type - TCG_TYPE_V64;
 177     TCGOP_VECE(op) = vece;
 178     op->args[0] = r;
 179     op->args[1] = a;
 180     op->args[2] = b;
 181     op->args[3] = c;
 182     op->args[4] = d;
 183     op->args[5] = e;
 184 }
 185
 186 static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a)
 187 {
 188     TCGTemp *rt = tcgv_vec_temp(r);
 189     TCGTemp *at = tcgv_vec_temp(a);
 190     TCGType type = rt->base_type;
 191
 192     /* Must enough inputs for the output.  */
 193     tcg_debug_assert(at->base_type >= type);
 194     vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at));
 195 }
 196
 197 static void vec_gen_op3(TCGOpcode opc, unsigned vece,
 198                         TCGv_vec r, TCGv_vec a, TCGv_vec b)
 199 {
 200     TCGTemp *rt = tcgv_vec_temp(r);
 201     TCGTemp *at = tcgv_vec_temp(a);
 202     TCGTemp *bt = tcgv_vec_temp(b);
 203     TCGType type = rt->base_type;
 204
 205     /* Must enough inputs for the output.  */
 206     tcg_debug_assert(at->base_type >= type);
 207     tcg_debug_assert(bt->base_type >= type);
 208     vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt));
 209 }
 210
 211 void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a)
 212 {
 213     if (r != a) {
 214         vec_gen_op2(INDEX_op_mov_vec, 0, r, a);
 215     }
 216 }
 217
 218 TCGv_vec tcg_const_zeros_vec(TCGType type)
 219 {
 220     TCGv_vec ret = tcg_temp_new_vec(type);
 221     tcg_gen_dupi_vec(MO_64, ret, 0);
 222     return ret;
 223 }
 224
 225 TCGv_vec tcg_const_ones_vec(TCGType type)
 226 {
 227     TCGv_vec ret = tcg_temp_new_vec(type);
 228     tcg_gen_dupi_vec(MO_64, ret, -1);
 229     return ret;
 230 }
 231
 232 TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec m)
 233 {
 234     TCGTemp *t = tcgv_vec_temp(m);
 235     return tcg_const_zeros_vec(t->base_type);
 236 }
 237
 238 TCGv_vec tcg_const_ones_vec_matching(TCGv_vec m)
 239 {
 240     TCGTemp *t = tcgv_vec_temp(m);
 241     return tcg_const_ones_vec(t->base_type);
 242 }
 243
 244 void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a)
 245 {
 246     TCGTemp *rt = tcgv_vec_temp(r);
 247     tcg_gen_mov_vec(r, tcg_constant_vec(rt->base_type, vece, a));
 248 }
 249
 250 void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a)
 251 {
 252     TCGArg ri = tcgv_vec_arg(r);
 253     TCGTemp *rt = arg_temp(ri);
 254     TCGType type = rt->base_type;
 255
 256     if (TCG_TARGET_REG_BITS == 64) {
 257         TCGArg ai = tcgv_i64_arg(a);
 258         vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
 259     } else if (vece == MO_64) {
 260         TCGArg al = tcgv_i32_arg(TCGV_LOW(a));
 261         TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a));
 262         vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah);
 263     } else {
 264         TCGArg ai = tcgv_i32_arg(TCGV_LOW(a));
 265         vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
 266     }
 267 }
 268
 269 void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a)
 270 {
 271     TCGArg ri = tcgv_vec_arg(r);
 272     TCGArg ai = tcgv_i32_arg(a);
 273     TCGTemp *rt = arg_temp(ri);
 274     TCGType type = rt->base_type;
 275
 276     vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
 277 }
 278
 279 void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec r, TCGv_ptr b,
 280                          tcg_target_long ofs)
 281 {
 282     TCGArg ri = tcgv_vec_arg(r);
 283     TCGArg bi = tcgv_ptr_arg(b);
 284     TCGTemp *rt = arg_temp(ri);
 285     TCGType type = rt->base_type;
 286
 287     vec_gen_3(INDEX_op_dupm_vec, type, vece, ri, bi, ofs);
 288 }
 289
 290 static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o)
 291 {
 292     TCGArg ri = tcgv_vec_arg(r);
 293     TCGArg bi = tcgv_ptr_arg(b);
 294     TCGTemp *rt = arg_temp(ri);
 295     TCGType type = rt->base_type;
 296
 297     vec_gen_3(opc, type, 0, ri, bi, o);
 298 }
 299
 300 void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
 301 {
 302     vec_gen_ldst(INDEX_op_ld_vec, r, b, o);
 303 }
 304
 305 void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
 306 {
 307     vec_gen_ldst(INDEX_op_st_vec, r, b, o);
 308 }
 309
 310 void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type)
 311 {
 312     TCGArg ri = tcgv_vec_arg(r);
 313     TCGArg bi = tcgv_ptr_arg(b);
 314     TCGTemp *rt = arg_temp(ri);
 315     TCGType type = rt->base_type;
 316
 317     tcg_debug_assert(low_type >= TCG_TYPE_V64);
 318     tcg_debug_assert(low_type <= type);
 319     vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o);
 320 }
 321
 322 void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 323 {
 324     vec_gen_op3(INDEX_op_and_vec, 0, r, a, b);
 325 }
 326
 327 void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 328 {
 329     vec_gen_op3(INDEX_op_or_vec, 0, r, a, b);
 330 }
 331
 332 void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 333 {
 334     vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b);
 335 }
 336
 337 void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 338 {
 339     if (TCG_TARGET_HAS_andc_vec) {
 340         vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b);
 341     } else {
 342         TCGv_vec t = tcg_temp_new_vec_matching(r);
 343         tcg_gen_not_vec(0, t, b);
 344         tcg_gen_and_vec(0, r, a, t);
 345         tcg_temp_free_vec(t);
 346     }
 347 }
 348
 349 void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 350 {
 351     if (TCG_TARGET_HAS_orc_vec) {
 352         vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b);
 353     } else {
 354         TCGv_vec t = tcg_temp_new_vec_matching(r);
 355         tcg_gen_not_vec(0, t, b);
 356         tcg_gen_or_vec(0, r, a, t);
 357         tcg_temp_free_vec(t);
 358     }
 359 }
 360
 361 void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 362 {
 363     /* TODO: Add TCG_TARGET_HAS_nand_vec when adding a backend supports it. */
 364     tcg_gen_and_vec(0, r, a, b);
 365     tcg_gen_not_vec(0, r, r);
 366 }
 367
 368 void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 369 {
 370     /* TODO: Add TCG_TARGET_HAS_nor_vec when adding a backend supports it. */
 371     tcg_gen_or_vec(0, r, a, b);
 372     tcg_gen_not_vec(0, r, r);
 373 }
 374
 375 void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 376 {
 377     /* TODO: Add TCG_TARGET_HAS_eqv_vec when adding a backend supports it. */
 378     tcg_gen_xor_vec(0, r, a, b);
 379     tcg_gen_not_vec(0, r, r);
 380 }
 381
 382 static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc)
 383 {
 384     TCGTemp *rt = tcgv_vec_temp(r);
 385     TCGTemp *at = tcgv_vec_temp(a);
 386     TCGArg ri = temp_arg(rt);
 387     TCGArg ai = temp_arg(at);
 388     TCGType type = rt->base_type;
 389     int can;
 390
 391     tcg_debug_assert(at->base_type >= type);
 392     tcg_assert_listed_vecop(opc);
 393     can = tcg_can_emit_vec_op(opc, type, vece);
 394     if (can > 0) {
 395         vec_gen_2(opc, type, vece, ri, ai);
 396     } else if (can < 0) {
 397         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 398         tcg_expand_vec_op(opc, type, vece, ri, ai);
 399         tcg_swap_vecop_list(hold_list);
 400     } else {
 401         return false;
 402     }
 403     return true;
 404 }
 405
 406 void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
 407 {
 408     const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 409
 410     if (!TCG_TARGET_HAS_not_vec || !do_op2(vece, r, a, INDEX_op_not_vec)) {
 411         TCGv_vec t = tcg_const_ones_vec_matching(r);
 412         tcg_gen_xor_vec(0, r, a, t);
 413         tcg_temp_free_vec(t);
 414     }
 415     tcg_swap_vecop_list(hold_list);
 416 }
 417
 418 void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
 419 {
 420     const TCGOpcode *hold_list;
 421
 422     tcg_assert_listed_vecop(INDEX_op_neg_vec);
 423     hold_list = tcg_swap_vecop_list(NULL);
 424
 425     if (!TCG_TARGET_HAS_neg_vec || !do_op2(vece, r, a, INDEX_op_neg_vec)) {
 426         TCGv_vec t = tcg_const_zeros_vec_matching(r);
 427         tcg_gen_sub_vec(vece, r, t, a);
 428         tcg_temp_free_vec(t);
 429     }
 430     tcg_swap_vecop_list(hold_list);
 431 }
 432
 433 void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
 434 {
 435     const TCGOpcode *hold_list;
 436
 437     tcg_assert_listed_vecop(INDEX_op_abs_vec);
 438     hold_list = tcg_swap_vecop_list(NULL);
 439
 440     if (!do_op2(vece, r, a, INDEX_op_abs_vec)) {
 441         TCGType type = tcgv_vec_temp(r)->base_type;
 442         TCGv_vec t = tcg_temp_new_vec(type);
 443
 444         tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece));
 445         if (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0) {
 446             tcg_gen_neg_vec(vece, t, a);
 447             tcg_gen_smax_vec(vece, r, a, t);
 448         } else {
 449             if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) {
 450                 tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1);
 451             } else {
 452                 tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a,
 453                                 tcg_constant_vec(type, vece, 0));
 454             }
 455             tcg_gen_xor_vec(vece, r, a, t);
 456             tcg_gen_sub_vec(vece, r, r, t);
 457         }
 458
 459         tcg_temp_free_vec(t);
 460     }
 461     tcg_swap_vecop_list(hold_list);
 462 }
 463
 464 static void do_shifti(TCGOpcode opc, unsigned vece,
 465                       TCGv_vec r, TCGv_vec a, int64_t i)
 466 {
 467     TCGTemp *rt = tcgv_vec_temp(r);
 468     TCGTemp *at = tcgv_vec_temp(a);
 469     TCGArg ri = temp_arg(rt);
 470     TCGArg ai = temp_arg(at);
 471     TCGType type = rt->base_type;
 472     int can;
 473
 474     tcg_debug_assert(at->base_type == type);
 475     tcg_debug_assert(i >= 0 && i < (8 << vece));
 476     tcg_assert_listed_vecop(opc);
 477
 478     if (i == 0) {
 479         tcg_gen_mov_vec(r, a);
 480         return;
 481     }
 482
 483     can = tcg_can_emit_vec_op(opc, type, vece);
 484     if (can > 0) {
 485         vec_gen_3(opc, type, vece, ri, ai, i);
 486     } else {
 487         /* We leave the choice of expansion via scalar or vector shift
 488            to the target.  Often, but not always, dupi can feed a vector
 489            shift easier than a scalar.  */
 490         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 491         tcg_debug_assert(can < 0);
 492         tcg_expand_vec_op(opc, type, vece, ri, ai, i);
 493         tcg_swap_vecop_list(hold_list);
 494     }
 495 }
 496
 497 void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
 498 {
 499     do_shifti(INDEX_op_shli_vec, vece, r, a, i);
 500 }
 501
 502 void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
 503 {
 504     do_shifti(INDEX_op_shri_vec, vece, r, a, i);
 505 }
 506
 507 void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
 508 {
 509     do_shifti(INDEX_op_sari_vec, vece, r, a, i);
 510 }
 511
 512 void tcg_gen_rotli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
 513 {
 514     do_shifti(INDEX_op_rotli_vec, vece, r, a, i);
 515 }
 516
 517 void tcg_gen_rotri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
 518 {
 519     int bits = 8 << vece;
 520     tcg_debug_assert(i >= 0 && i < bits);
 521     do_shifti(INDEX_op_rotli_vec, vece, r, a, -i & (bits - 1));
 522 }
 523
 524 void tcg_gen_cmp_vec(TCGCond cond, unsigned vece,
 525                      TCGv_vec r, TCGv_vec a, TCGv_vec b)
 526 {
 527     TCGTemp *rt = tcgv_vec_temp(r);
 528     TCGTemp *at = tcgv_vec_temp(a);
 529     TCGTemp *bt = tcgv_vec_temp(b);
 530     TCGArg ri = temp_arg(rt);
 531     TCGArg ai = temp_arg(at);
 532     TCGArg bi = temp_arg(bt);
 533     TCGType type = rt->base_type;
 534     int can;
 535
 536     tcg_debug_assert(at->base_type >= type);
 537     tcg_debug_assert(bt->base_type >= type);
 538     tcg_assert_listed_vecop(INDEX_op_cmp_vec);
 539     can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece);
 540     if (can > 0) {
 541         vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
 542     } else {
 543         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 544         tcg_debug_assert(can < 0);
 545         tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
 546         tcg_swap_vecop_list(hold_list);
 547     }
 548 }
 549
 550 static bool do_op3(unsigned vece, TCGv_vec r, TCGv_vec a,
 551                    TCGv_vec b, TCGOpcode opc)
 552 {
 553     TCGTemp *rt = tcgv_vec_temp(r);
 554     TCGTemp *at = tcgv_vec_temp(a);
 555     TCGTemp *bt = tcgv_vec_temp(b);
 556     TCGArg ri = temp_arg(rt);
 557     TCGArg ai = temp_arg(at);
 558     TCGArg bi = temp_arg(bt);
 559     TCGType type = rt->base_type;
 560     int can;
 561
 562     tcg_debug_assert(at->base_type >= type);
 563     tcg_debug_assert(bt->base_type >= type);
 564     tcg_assert_listed_vecop(opc);
 565     can = tcg_can_emit_vec_op(opc, type, vece);
 566     if (can > 0) {
 567         vec_gen_3(opc, type, vece, ri, ai, bi);
 568     } else if (can < 0) {
 569         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 570         tcg_expand_vec_op(opc, type, vece, ri, ai, bi);
 571         tcg_swap_vecop_list(hold_list);
 572     } else {
 573         return false;
 574     }
 575     return true;
 576 }
 577
 578 static void do_op3_nofail(unsigned vece, TCGv_vec r, TCGv_vec a,
 579                           TCGv_vec b, TCGOpcode opc)
 580 {
 581     bool ok = do_op3(vece, r, a, b, opc);
 582     tcg_debug_assert(ok);
 583 }
 584
 585 void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 586 {
 587     do_op3_nofail(vece, r, a, b, INDEX_op_add_vec);
 588 }
 589
 590 void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 591 {
 592     do_op3_nofail(vece, r, a, b, INDEX_op_sub_vec);
 593 }
 594
 595 void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 596 {
 597     do_op3_nofail(vece, r, a, b, INDEX_op_mul_vec);
 598 }
 599
 600 void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 601 {
 602     do_op3_nofail(vece, r, a, b, INDEX_op_ssadd_vec);
 603 }
 604
 605 void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 606 {
 607     do_op3_nofail(vece, r, a, b, INDEX_op_usadd_vec);
 608 }
 609
 610 void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 611 {
 612     do_op3_nofail(vece, r, a, b, INDEX_op_sssub_vec);
 613 }
 614
 615 void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 616 {
 617     do_op3_nofail(vece, r, a, b, INDEX_op_ussub_vec);
 618 }
 619
 620 static void do_minmax(unsigned vece, TCGv_vec r, TCGv_vec a,
 621                       TCGv_vec b, TCGOpcode opc, TCGCond cond)
 622 {
 623     if (!do_op3(vece, r, a, b, opc)) {
 624         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 625         tcg_gen_cmpsel_vec(cond, vece, r, a, b, a, b);
 626         tcg_swap_vecop_list(hold_list);
 627     }
 628 }
 629
 630 void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 631 {
 632     do_minmax(vece, r, a, b, INDEX_op_smin_vec, TCG_COND_LT);
 633 }
 634
 635 void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 636 {
 637     do_minmax(vece, r, a, b, INDEX_op_umin_vec, TCG_COND_LTU);
 638 }
 639
 640 void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 641 {
 642     do_minmax(vece, r, a, b, INDEX_op_smax_vec, TCG_COND_GT);
 643 }
 644
 645 void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 646 {
 647     do_minmax(vece, r, a, b, INDEX_op_umax_vec, TCG_COND_GTU);
 648 }
 649
 650 void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 651 {
 652     do_op3_nofail(vece, r, a, b, INDEX_op_shlv_vec);
 653 }
 654
 655 void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 656 {
 657     do_op3_nofail(vece, r, a, b, INDEX_op_shrv_vec);
 658 }
 659
 660 void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 661 {
 662     do_op3_nofail(vece, r, a, b, INDEX_op_sarv_vec);
 663 }
 664
 665 void tcg_gen_rotlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 666 {
 667     do_op3_nofail(vece, r, a, b, INDEX_op_rotlv_vec);
 668 }
 669
 670 void tcg_gen_rotrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 671 {
 672     do_op3_nofail(vece, r, a, b, INDEX_op_rotrv_vec);
 673 }
 674
 675 static void do_shifts(unsigned vece, TCGv_vec r, TCGv_vec a,
 676                       TCGv_i32 s, TCGOpcode opc)
 677 {
 678     TCGTemp *rt = tcgv_vec_temp(r);
 679     TCGTemp *at = tcgv_vec_temp(a);
 680     TCGTemp *st = tcgv_i32_temp(s);
 681     TCGArg ri = temp_arg(rt);
 682     TCGArg ai = temp_arg(at);
 683     TCGArg si = temp_arg(st);
 684     TCGType type = rt->base_type;
 685     int can;
 686
 687     tcg_debug_assert(at->base_type >= type);
 688     tcg_assert_listed_vecop(opc);
 689     can = tcg_can_emit_vec_op(opc, type, vece);
 690     if (can > 0) {
 691         vec_gen_3(opc, type, vece, ri, ai, si);
 692     } else if (can < 0) {
 693         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 694         tcg_expand_vec_op(opc, type, vece, ri, ai, si);
 695         tcg_swap_vecop_list(hold_list);
 696     } else {
 697         g_assert_not_reached();
 698     }
 699 }
 700
 701 void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
 702 {
 703     do_shifts(vece, r, a, b, INDEX_op_shls_vec);
 704 }
 705
 706 void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
 707 {
 708     do_shifts(vece, r, a, b, INDEX_op_shrs_vec);
 709 }
 710
 711 void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
 712 {
 713     do_shifts(vece, r, a, b, INDEX_op_sars_vec);
 714 }
 715
 716 void tcg_gen_rotls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s)
 717 {
 718     do_shifts(vece, r, a, s, INDEX_op_rotls_vec);
 719 }
 720
 721 void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a,
 722                         TCGv_vec b, TCGv_vec c)
 723 {
 724     TCGTemp *rt = tcgv_vec_temp(r);
 725     TCGTemp *at = tcgv_vec_temp(a);
 726     TCGTemp *bt = tcgv_vec_temp(b);
 727     TCGTemp *ct = tcgv_vec_temp(c);
 728     TCGType type = rt->base_type;
 729
 730     tcg_debug_assert(at->base_type >= type);
 731     tcg_debug_assert(bt->base_type >= type);
 732     tcg_debug_assert(ct->base_type >= type);
 733
 734     if (TCG_TARGET_HAS_bitsel_vec) {
 735         vec_gen_4(INDEX_op_bitsel_vec, type, MO_8,
 736                   temp_arg(rt), temp_arg(at), temp_arg(bt), temp_arg(ct));
 737     } else {
 738         TCGv_vec t = tcg_temp_new_vec(type);
 739         tcg_gen_and_vec(MO_8, t, a, b);
 740         tcg_gen_andc_vec(MO_8, r, c, a);
 741         tcg_gen_or_vec(MO_8, r, r, t);
 742         tcg_temp_free_vec(t);
 743     }
 744 }
 745
 746 void tcg_gen_cmpsel_vec(TCGCond cond, unsigned vece, TCGv_vec r,
 747                         TCGv_vec a, TCGv_vec b, TCGv_vec c, TCGv_vec d)
 748 {
 749     TCGTemp *rt = tcgv_vec_temp(r);
 750     TCGTemp *at = tcgv_vec_temp(a);
 751     TCGTemp *bt = tcgv_vec_temp(b);
 752     TCGTemp *ct = tcgv_vec_temp(c);
 753     TCGTemp *dt = tcgv_vec_temp(d);
 754     TCGArg ri = temp_arg(rt);
 755     TCGArg ai = temp_arg(at);
 756     TCGArg bi = temp_arg(bt);
 757     TCGArg ci = temp_arg(ct);
 758     TCGArg di = temp_arg(dt);
 759     TCGType type = rt->base_type;
 760     const TCGOpcode *hold_list;
 761     int can;
 762
 763     tcg_debug_assert(at->base_type >= type);
 764     tcg_debug_assert(bt->base_type >= type);
 765     tcg_debug_assert(ct->base_type >= type);
 766     tcg_debug_assert(dt->base_type >= type);
 767
 768     tcg_assert_listed_vecop(INDEX_op_cmpsel_vec);
 769     hold_list = tcg_swap_vecop_list(NULL);
 770     can = tcg_can_emit_vec_op(INDEX_op_cmpsel_vec, type, vece);
 771
 772     if (can > 0) {
 773         vec_gen_6(INDEX_op_cmpsel_vec, type, vece, ri, ai, bi, ci, di, cond);
 774     } else if (can < 0) {
 775         tcg_expand_vec_op(INDEX_op_cmpsel_vec, type, vece,
 776                           ri, ai, bi, ci, di, cond);
 777     } else {
 778         TCGv_vec t = tcg_temp_new_vec(type);
 779         tcg_gen_cmp_vec(cond, vece, t, a, b);
 780         tcg_gen_bitsel_vec(vece, r, t, c, d);
 781         tcg_temp_free_vec(t);
 782     }
 783     tcg_swap_vecop_list(hold_list);
 784 }