tcg/tcg-op-vec.c

   1 /*
   2  * Tiny Code Generator for QEMU
   3  *
   4  * Copyright (c) 2018 Linaro, Inc.
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "qemu/osdep.h"
  21 #include "tcg/tcg.h"
  22 #include "tcg/tcg-op.h"
  23 #include "tcg/tcg-mo.h"
  24
  25 /* Reduce the number of ifdefs below.  This assumes that all uses of
  26    TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
  27    the compiler can eliminate.  */
  28 #if TCG_TARGET_REG_BITS == 64
  29 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
  30 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
  31 #define TCGV_LOW  TCGV_LOW_link_error
  32 #define TCGV_HIGH TCGV_HIGH_link_error
  33 #endif
  34
  35 /*
  36  * Vector optional opcode tracking.
  37  * Except for the basic logical operations (and, or, xor), and
  38  * data movement (mov, ld, st, dupi), many vector opcodes are
  39  * optional and may not be supported on the host.  Thank Intel
  40  * for the irregularity in their instruction set.
  41  *
  42  * The gvec expanders allow custom vector operations to be composed,
  43  * generally via the .fniv callback in the GVecGen* structures.  At
  44  * the same time, in deciding whether to use this hook we need to
  45  * know if the host supports the required operations.  This is
  46  * presented as an array of opcodes, terminated by 0.  Each opcode
  47  * is assumed to be expanded with the given VECE.
  48  *
  49  * For debugging, we want to validate this array.  Therefore, when
  50  * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders
  51  * will validate that their opcode is present in the list.
  52  */
  53 #ifdef CONFIG_DEBUG_TCG
  54 void tcg_assert_listed_vecop(TCGOpcode op)
  55 {
  56     const TCGOpcode *p = tcg_ctx->vecop_list;
  57     if (p) {
  58         for (; *p; ++p) {
  59             if (*p == op) {
  60                 return;
  61             }
  62         }
  63         g_assert_not_reached();
  64     }
  65 }
  66 #endif
  67
  68 bool tcg_can_emit_vecop_list(const TCGOpcode *list,
  69                              TCGType type, unsigned vece)
  70 {
  71     if (list == NULL) {
  72         return true;
  73     }
  74
  75     for (; *list; ++list) {
  76         TCGOpcode opc = *list;
  77
  78 #ifdef CONFIG_DEBUG_TCG
  79         switch (opc) {
  80         case INDEX_op_and_vec:
  81         case INDEX_op_or_vec:
  82         case INDEX_op_xor_vec:
  83         case INDEX_op_mov_vec:
  84         case INDEX_op_dup_vec:
  85         case INDEX_op_dup2_vec:
  86         case INDEX_op_ld_vec:
  87         case INDEX_op_st_vec:
  88         case INDEX_op_bitsel_vec:
  89             /* These opcodes are mandatory and should not be listed.  */
  90             g_assert_not_reached();
  91         case INDEX_op_not_vec:
  92             /* These opcodes have generic expansions using the above.  */
  93             g_assert_not_reached();
  94         default:
  95             break;
  96         }
  97 #endif
  98
  99         if (tcg_can_emit_vec_op(opc, type, vece)) {
 100             continue;
 101         }
 102
 103         /*
 104          * The opcode list is created by front ends based on what they
 105          * actually invoke.  We must mirror the logic in the routines
 106          * below for generic expansions using other opcodes.
 107          */
 108         switch (opc) {
 109         case INDEX_op_neg_vec:
 110             if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)) {
 111                 continue;
 112             }
 113             break;
 114         case INDEX_op_abs_vec:
 115             if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)
 116                 && (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0
 117                     || tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0
 118                     || tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece))) {
 119                 continue;
 120             }
 121             break;
 122         case INDEX_op_cmpsel_vec:
 123         case INDEX_op_smin_vec:
 124         case INDEX_op_smax_vec:
 125         case INDEX_op_umin_vec:
 126         case INDEX_op_umax_vec:
 127             if (tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
 128                 continue;
 129             }
 130             break;
 131         default:
 132             break;
 133         }
 134         return false;
 135     }
 136     return true;
 137 }
 138
 139 void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a)
 140 {
 141     TCGOp *op = tcg_emit_op(opc);
 142     TCGOP_VECL(op) = type - TCG_TYPE_V64;
 143     TCGOP_VECE(op) = vece;
 144     op->args[0] = r;
 145     op->args[1] = a;
 146 }
 147
 148 void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece,
 149                TCGArg r, TCGArg a, TCGArg b)
 150 {
 151     TCGOp *op = tcg_emit_op(opc);
 152     TCGOP_VECL(op) = type - TCG_TYPE_V64;
 153     TCGOP_VECE(op) = vece;
 154     op->args[0] = r;
 155     op->args[1] = a;
 156     op->args[2] = b;
 157 }
 158
 159 void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece,
 160                TCGArg r, TCGArg a, TCGArg b, TCGArg c)
 161 {
 162     TCGOp *op = tcg_emit_op(opc);
 163     TCGOP_VECL(op) = type - TCG_TYPE_V64;
 164     TCGOP_VECE(op) = vece;
 165     op->args[0] = r;
 166     op->args[1] = a;
 167     op->args[2] = b;
 168     op->args[3] = c;
 169 }
 170
 171 static void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r,
 172                       TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e)
 173 {
 174     TCGOp *op = tcg_emit_op(opc);
 175     TCGOP_VECL(op) = type - TCG_TYPE_V64;
 176     TCGOP_VECE(op) = vece;
 177     op->args[0] = r;
 178     op->args[1] = a;
 179     op->args[2] = b;
 180     op->args[3] = c;
 181     op->args[4] = d;
 182     op->args[5] = e;
 183 }
 184
 185 static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a)
 186 {
 187     TCGTemp *rt = tcgv_vec_temp(r);
 188     TCGTemp *at = tcgv_vec_temp(a);
 189     TCGType type = rt->base_type;
 190
 191     /* Must enough inputs for the output.  */
 192     tcg_debug_assert(at->base_type >= type);
 193     vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at));
 194 }
 195
 196 static void vec_gen_op3(TCGOpcode opc, unsigned vece,
 197                         TCGv_vec r, TCGv_vec a, TCGv_vec b)
 198 {
 199     TCGTemp *rt = tcgv_vec_temp(r);
 200     TCGTemp *at = tcgv_vec_temp(a);
 201     TCGTemp *bt = tcgv_vec_temp(b);
 202     TCGType type = rt->base_type;
 203
 204     /* Must enough inputs for the output.  */
 205     tcg_debug_assert(at->base_type >= type);
 206     tcg_debug_assert(bt->base_type >= type);
 207     vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt));
 208 }
 209
 210 void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a)
 211 {
 212     if (r != a) {
 213         vec_gen_op2(INDEX_op_mov_vec, 0, r, a);
 214     }
 215 }
 216
 217 TCGv_vec tcg_const_zeros_vec(TCGType type)
 218 {
 219     TCGv_vec ret = tcg_temp_new_vec(type);
 220     tcg_gen_dupi_vec(MO_64, ret, 0);
 221     return ret;
 222 }
 223
 224 TCGv_vec tcg_const_ones_vec(TCGType type)
 225 {
 226     TCGv_vec ret = tcg_temp_new_vec(type);
 227     tcg_gen_dupi_vec(MO_64, ret, -1);
 228     return ret;
 229 }
 230
 231 TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec m)
 232 {
 233     TCGTemp *t = tcgv_vec_temp(m);
 234     return tcg_const_zeros_vec(t->base_type);
 235 }
 236
 237 TCGv_vec tcg_const_ones_vec_matching(TCGv_vec m)
 238 {
 239     TCGTemp *t = tcgv_vec_temp(m);
 240     return tcg_const_ones_vec(t->base_type);
 241 }
 242
 243 void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a)
 244 {
 245     TCGTemp *rt = tcgv_vec_temp(r);
 246     tcg_gen_mov_vec(r, tcg_constant_vec(rt->base_type, vece, a));
 247 }
 248
 249 void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a)
 250 {
 251     TCGArg ri = tcgv_vec_arg(r);
 252     TCGTemp *rt = arg_temp(ri);
 253     TCGType type = rt->base_type;
 254
 255     if (TCG_TARGET_REG_BITS == 64) {
 256         TCGArg ai = tcgv_i64_arg(a);
 257         vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
 258     } else if (vece == MO_64) {
 259         TCGArg al = tcgv_i32_arg(TCGV_LOW(a));
 260         TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a));
 261         vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah);
 262     } else {
 263         TCGArg ai = tcgv_i32_arg(TCGV_LOW(a));
 264         vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
 265     }
 266 }
 267
 268 void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a)
 269 {
 270     TCGArg ri = tcgv_vec_arg(r);
 271     TCGArg ai = tcgv_i32_arg(a);
 272     TCGTemp *rt = arg_temp(ri);
 273     TCGType type = rt->base_type;
 274
 275     vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
 276 }
 277
 278 void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec r, TCGv_ptr b,
 279                          tcg_target_long ofs)
 280 {
 281     TCGArg ri = tcgv_vec_arg(r);
 282     TCGArg bi = tcgv_ptr_arg(b);
 283     TCGTemp *rt = arg_temp(ri);
 284     TCGType type = rt->base_type;
 285
 286     vec_gen_3(INDEX_op_dupm_vec, type, vece, ri, bi, ofs);
 287 }
 288
 289 static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o)
 290 {
 291     TCGArg ri = tcgv_vec_arg(r);
 292     TCGArg bi = tcgv_ptr_arg(b);
 293     TCGTemp *rt = arg_temp(ri);
 294     TCGType type = rt->base_type;
 295
 296     vec_gen_3(opc, type, 0, ri, bi, o);
 297 }
 298
 299 void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
 300 {
 301     vec_gen_ldst(INDEX_op_ld_vec, r, b, o);
 302 }
 303
 304 void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
 305 {
 306     vec_gen_ldst(INDEX_op_st_vec, r, b, o);
 307 }
 308
 309 void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type)
 310 {
 311     TCGArg ri = tcgv_vec_arg(r);
 312     TCGArg bi = tcgv_ptr_arg(b);
 313     TCGTemp *rt = arg_temp(ri);
 314     TCGType type = rt->base_type;
 315
 316     tcg_debug_assert(low_type >= TCG_TYPE_V64);
 317     tcg_debug_assert(low_type <= type);
 318     vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o);
 319 }
 320
 321 void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 322 {
 323     vec_gen_op3(INDEX_op_and_vec, 0, r, a, b);
 324 }
 325
 326 void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 327 {
 328     vec_gen_op3(INDEX_op_or_vec, 0, r, a, b);
 329 }
 330
 331 void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 332 {
 333     vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b);
 334 }
 335
 336 void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 337 {
 338     if (TCG_TARGET_HAS_andc_vec) {
 339         vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b);
 340     } else {
 341         TCGv_vec t = tcg_temp_new_vec_matching(r);
 342         tcg_gen_not_vec(0, t, b);
 343         tcg_gen_and_vec(0, r, a, t);
 344         tcg_temp_free_vec(t);
 345     }
 346 }
 347
 348 void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 349 {
 350     if (TCG_TARGET_HAS_orc_vec) {
 351         vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b);
 352     } else {
 353         TCGv_vec t = tcg_temp_new_vec_matching(r);
 354         tcg_gen_not_vec(0, t, b);
 355         tcg_gen_or_vec(0, r, a, t);
 356         tcg_temp_free_vec(t);
 357     }
 358 }
 359
 360 void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 361 {
 362     /* TODO: Add TCG_TARGET_HAS_nand_vec when adding a backend supports it. */
 363     tcg_gen_and_vec(0, r, a, b);
 364     tcg_gen_not_vec(0, r, r);
 365 }
 366
 367 void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 368 {
 369     /* TODO: Add TCG_TARGET_HAS_nor_vec when adding a backend supports it. */
 370     tcg_gen_or_vec(0, r, a, b);
 371     tcg_gen_not_vec(0, r, r);
 372 }
 373
 374 void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 375 {
 376     /* TODO: Add TCG_TARGET_HAS_eqv_vec when adding a backend supports it. */
 377     tcg_gen_xor_vec(0, r, a, b);
 378     tcg_gen_not_vec(0, r, r);
 379 }
 380
 381 static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc)
 382 {
 383     TCGTemp *rt = tcgv_vec_temp(r);
 384     TCGTemp *at = tcgv_vec_temp(a);
 385     TCGArg ri = temp_arg(rt);
 386     TCGArg ai = temp_arg(at);
 387     TCGType type = rt->base_type;
 388     int can;
 389
 390     tcg_debug_assert(at->base_type >= type);
 391     tcg_assert_listed_vecop(opc);
 392     can = tcg_can_emit_vec_op(opc, type, vece);
 393     if (can > 0) {
 394         vec_gen_2(opc, type, vece, ri, ai);
 395     } else if (can < 0) {
 396         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 397         tcg_expand_vec_op(opc, type, vece, ri, ai);
 398         tcg_swap_vecop_list(hold_list);
 399     } else {
 400         return false;
 401     }
 402     return true;
 403 }
 404
 405 void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
 406 {
 407     const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 408
 409     if (!TCG_TARGET_HAS_not_vec || !do_op2(vece, r, a, INDEX_op_not_vec)) {
 410         TCGv_vec t = tcg_const_ones_vec_matching(r);
 411         tcg_gen_xor_vec(0, r, a, t);
 412         tcg_temp_free_vec(t);
 413     }
 414     tcg_swap_vecop_list(hold_list);
 415 }
 416
 417 void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
 418 {
 419     const TCGOpcode *hold_list;
 420
 421     tcg_assert_listed_vecop(INDEX_op_neg_vec);
 422     hold_list = tcg_swap_vecop_list(NULL);
 423
 424     if (!TCG_TARGET_HAS_neg_vec || !do_op2(vece, r, a, INDEX_op_neg_vec)) {
 425         TCGv_vec t = tcg_const_zeros_vec_matching(r);
 426         tcg_gen_sub_vec(vece, r, t, a);
 427         tcg_temp_free_vec(t);
 428     }
 429     tcg_swap_vecop_list(hold_list);
 430 }
 431
 432 void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
 433 {
 434     const TCGOpcode *hold_list;
 435
 436     tcg_assert_listed_vecop(INDEX_op_abs_vec);
 437     hold_list = tcg_swap_vecop_list(NULL);
 438
 439     if (!do_op2(vece, r, a, INDEX_op_abs_vec)) {
 440         TCGType type = tcgv_vec_temp(r)->base_type;
 441         TCGv_vec t = tcg_temp_new_vec(type);
 442
 443         tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece));
 444         if (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0) {
 445             tcg_gen_neg_vec(vece, t, a);
 446             tcg_gen_smax_vec(vece, r, a, t);
 447         } else {
 448             if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) {
 449                 tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1);
 450             } else {
 451                 tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a,
 452                                 tcg_constant_vec(type, vece, 0));
 453             }
 454             tcg_gen_xor_vec(vece, r, a, t);
 455             tcg_gen_sub_vec(vece, r, r, t);
 456         }
 457
 458         tcg_temp_free_vec(t);
 459     }
 460     tcg_swap_vecop_list(hold_list);
 461 }
 462
 463 static void do_shifti(TCGOpcode opc, unsigned vece,
 464                       TCGv_vec r, TCGv_vec a, int64_t i)
 465 {
 466     TCGTemp *rt = tcgv_vec_temp(r);
 467     TCGTemp *at = tcgv_vec_temp(a);
 468     TCGArg ri = temp_arg(rt);
 469     TCGArg ai = temp_arg(at);
 470     TCGType type = rt->base_type;
 471     int can;
 472
 473     tcg_debug_assert(at->base_type == type);
 474     tcg_debug_assert(i >= 0 && i < (8 << vece));
 475     tcg_assert_listed_vecop(opc);
 476
 477     if (i == 0) {
 478         tcg_gen_mov_vec(r, a);
 479         return;
 480     }
 481
 482     can = tcg_can_emit_vec_op(opc, type, vece);
 483     if (can > 0) {
 484         vec_gen_3(opc, type, vece, ri, ai, i);
 485     } else {
 486         /* We leave the choice of expansion via scalar or vector shift
 487            to the target.  Often, but not always, dupi can feed a vector
 488            shift easier than a scalar.  */
 489         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 490         tcg_debug_assert(can < 0);
 491         tcg_expand_vec_op(opc, type, vece, ri, ai, i);
 492         tcg_swap_vecop_list(hold_list);
 493     }
 494 }
 495
 496 void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
 497 {
 498     do_shifti(INDEX_op_shli_vec, vece, r, a, i);
 499 }
 500
 501 void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
 502 {
 503     do_shifti(INDEX_op_shri_vec, vece, r, a, i);
 504 }
 505
 506 void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
 507 {
 508     do_shifti(INDEX_op_sari_vec, vece, r, a, i);
 509 }
 510
 511 void tcg_gen_rotli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
 512 {
 513     do_shifti(INDEX_op_rotli_vec, vece, r, a, i);
 514 }
 515
 516 void tcg_gen_rotri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
 517 {
 518     int bits = 8 << vece;
 519     tcg_debug_assert(i >= 0 && i < bits);
 520     do_shifti(INDEX_op_rotli_vec, vece, r, a, -i & (bits - 1));
 521 }
 522
 523 void tcg_gen_cmp_vec(TCGCond cond, unsigned vece,
 524                      TCGv_vec r, TCGv_vec a, TCGv_vec b)
 525 {
 526     TCGTemp *rt = tcgv_vec_temp(r);
 527     TCGTemp *at = tcgv_vec_temp(a);
 528     TCGTemp *bt = tcgv_vec_temp(b);
 529     TCGArg ri = temp_arg(rt);
 530     TCGArg ai = temp_arg(at);
 531     TCGArg bi = temp_arg(bt);
 532     TCGType type = rt->base_type;
 533     int can;
 534
 535     tcg_debug_assert(at->base_type >= type);
 536     tcg_debug_assert(bt->base_type >= type);
 537     tcg_assert_listed_vecop(INDEX_op_cmp_vec);
 538     can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece);
 539     if (can > 0) {
 540         vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
 541     } else {
 542         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 543         tcg_debug_assert(can < 0);
 544         tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
 545         tcg_swap_vecop_list(hold_list);
 546     }
 547 }
 548
 549 static bool do_op3(unsigned vece, TCGv_vec r, TCGv_vec a,
 550                    TCGv_vec b, TCGOpcode opc)
 551 {
 552     TCGTemp *rt = tcgv_vec_temp(r);
 553     TCGTemp *at = tcgv_vec_temp(a);
 554     TCGTemp *bt = tcgv_vec_temp(b);
 555     TCGArg ri = temp_arg(rt);
 556     TCGArg ai = temp_arg(at);
 557     TCGArg bi = temp_arg(bt);
 558     TCGType type = rt->base_type;
 559     int can;
 560
 561     tcg_debug_assert(at->base_type >= type);
 562     tcg_debug_assert(bt->base_type >= type);
 563     tcg_assert_listed_vecop(opc);
 564     can = tcg_can_emit_vec_op(opc, type, vece);
 565     if (can > 0) {
 566         vec_gen_3(opc, type, vece, ri, ai, bi);
 567     } else if (can < 0) {
 568         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 569         tcg_expand_vec_op(opc, type, vece, ri, ai, bi);
 570         tcg_swap_vecop_list(hold_list);
 571     } else {
 572         return false;
 573     }
 574     return true;
 575 }
 576
 577 static void do_op3_nofail(unsigned vece, TCGv_vec r, TCGv_vec a,
 578                           TCGv_vec b, TCGOpcode opc)
 579 {
 580     bool ok = do_op3(vece, r, a, b, opc);
 581     tcg_debug_assert(ok);
 582 }
 583
 584 void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 585 {
 586     do_op3_nofail(vece, r, a, b, INDEX_op_add_vec);
 587 }
 588
 589 void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 590 {
 591     do_op3_nofail(vece, r, a, b, INDEX_op_sub_vec);
 592 }
 593
 594 void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 595 {
 596     do_op3_nofail(vece, r, a, b, INDEX_op_mul_vec);
 597 }
 598
 599 void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 600 {
 601     do_op3_nofail(vece, r, a, b, INDEX_op_ssadd_vec);
 602 }
 603
 604 void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 605 {
 606     do_op3_nofail(vece, r, a, b, INDEX_op_usadd_vec);
 607 }
 608
 609 void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 610 {
 611     do_op3_nofail(vece, r, a, b, INDEX_op_sssub_vec);
 612 }
 613
 614 void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 615 {
 616     do_op3_nofail(vece, r, a, b, INDEX_op_ussub_vec);
 617 }
 618
 619 static void do_minmax(unsigned vece, TCGv_vec r, TCGv_vec a,
 620                       TCGv_vec b, TCGOpcode opc, TCGCond cond)
 621 {
 622     if (!do_op3(vece, r, a, b, opc)) {
 623         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 624         tcg_gen_cmpsel_vec(cond, vece, r, a, b, a, b);
 625         tcg_swap_vecop_list(hold_list);
 626     }
 627 }
 628
 629 void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 630 {
 631     do_minmax(vece, r, a, b, INDEX_op_smin_vec, TCG_COND_LT);
 632 }
 633
 634 void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 635 {
 636     do_minmax(vece, r, a, b, INDEX_op_umin_vec, TCG_COND_LTU);
 637 }
 638
 639 void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 640 {
 641     do_minmax(vece, r, a, b, INDEX_op_smax_vec, TCG_COND_GT);
 642 }
 643
 644 void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 645 {
 646     do_minmax(vece, r, a, b, INDEX_op_umax_vec, TCG_COND_GTU);
 647 }
 648
 649 void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 650 {
 651     do_op3_nofail(vece, r, a, b, INDEX_op_shlv_vec);
 652 }
 653
 654 void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 655 {
 656     do_op3_nofail(vece, r, a, b, INDEX_op_shrv_vec);
 657 }
 658
 659 void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 660 {
 661     do_op3_nofail(vece, r, a, b, INDEX_op_sarv_vec);
 662 }
 663
 664 void tcg_gen_rotlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 665 {
 666     do_op3_nofail(vece, r, a, b, INDEX_op_rotlv_vec);
 667 }
 668
 669 void tcg_gen_rotrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 670 {
 671     do_op3_nofail(vece, r, a, b, INDEX_op_rotrv_vec);
 672 }
 673
 674 static void do_shifts(unsigned vece, TCGv_vec r, TCGv_vec a,
 675                       TCGv_i32 s, TCGOpcode opc)
 676 {
 677     TCGTemp *rt = tcgv_vec_temp(r);
 678     TCGTemp *at = tcgv_vec_temp(a);
 679     TCGTemp *st = tcgv_i32_temp(s);
 680     TCGArg ri = temp_arg(rt);
 681     TCGArg ai = temp_arg(at);
 682     TCGArg si = temp_arg(st);
 683     TCGType type = rt->base_type;
 684     int can;
 685
 686     tcg_debug_assert(at->base_type >= type);
 687     tcg_assert_listed_vecop(opc);
 688     can = tcg_can_emit_vec_op(opc, type, vece);
 689     if (can > 0) {
 690         vec_gen_3(opc, type, vece, ri, ai, si);
 691     } else if (can < 0) {
 692         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 693         tcg_expand_vec_op(opc, type, vece, ri, ai, si);
 694         tcg_swap_vecop_list(hold_list);
 695     } else {
 696         g_assert_not_reached();
 697     }
 698 }
 699
 700 void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
 701 {
 702     do_shifts(vece, r, a, b, INDEX_op_shls_vec);
 703 }
 704
 705 void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
 706 {
 707     do_shifts(vece, r, a, b, INDEX_op_shrs_vec);
 708 }
 709
 710 void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
 711 {
 712     do_shifts(vece, r, a, b, INDEX_op_sars_vec);
 713 }
 714
 715 void tcg_gen_rotls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s)
 716 {
 717     do_shifts(vece, r, a, s, INDEX_op_rotls_vec);
 718 }
 719
 720 void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a,
 721                         TCGv_vec b, TCGv_vec c)
 722 {
 723     TCGTemp *rt = tcgv_vec_temp(r);
 724     TCGTemp *at = tcgv_vec_temp(a);
 725     TCGTemp *bt = tcgv_vec_temp(b);
 726     TCGTemp *ct = tcgv_vec_temp(c);
 727     TCGType type = rt->base_type;
 728
 729     tcg_debug_assert(at->base_type >= type);
 730     tcg_debug_assert(bt->base_type >= type);
 731     tcg_debug_assert(ct->base_type >= type);
 732
 733     if (TCG_TARGET_HAS_bitsel_vec) {
 734         vec_gen_4(INDEX_op_bitsel_vec, type, MO_8,
 735                   temp_arg(rt), temp_arg(at), temp_arg(bt), temp_arg(ct));
 736     } else {
 737         TCGv_vec t = tcg_temp_new_vec(type);
 738         tcg_gen_and_vec(MO_8, t, a, b);
 739         tcg_gen_andc_vec(MO_8, r, c, a);
 740         tcg_gen_or_vec(MO_8, r, r, t);
 741         tcg_temp_free_vec(t);
 742     }
 743 }
 744
 745 void tcg_gen_cmpsel_vec(TCGCond cond, unsigned vece, TCGv_vec r,
 746                         TCGv_vec a, TCGv_vec b, TCGv_vec c, TCGv_vec d)
 747 {
 748     TCGTemp *rt = tcgv_vec_temp(r);
 749     TCGTemp *at = tcgv_vec_temp(a);
 750     TCGTemp *bt = tcgv_vec_temp(b);
 751     TCGTemp *ct = tcgv_vec_temp(c);
 752     TCGTemp *dt = tcgv_vec_temp(d);
 753     TCGArg ri = temp_arg(rt);
 754     TCGArg ai = temp_arg(at);
 755     TCGArg bi = temp_arg(bt);
 756     TCGArg ci = temp_arg(ct);
 757     TCGArg di = temp_arg(dt);
 758     TCGType type = rt->base_type;
 759     const TCGOpcode *hold_list;
 760     int can;
 761
 762     tcg_debug_assert(at->base_type >= type);
 763     tcg_debug_assert(bt->base_type >= type);
 764     tcg_debug_assert(ct->base_type >= type);
 765     tcg_debug_assert(dt->base_type >= type);
 766
 767     tcg_assert_listed_vecop(INDEX_op_cmpsel_vec);
 768     hold_list = tcg_swap_vecop_list(NULL);
 769     can = tcg_can_emit_vec_op(INDEX_op_cmpsel_vec, type, vece);
 770
 771     if (can > 0) {
 772         vec_gen_6(INDEX_op_cmpsel_vec, type, vece, ri, ai, bi, ci, di, cond);
 773     } else if (can < 0) {
 774         tcg_expand_vec_op(INDEX_op_cmpsel_vec, type, vece,
 775                           ri, ai, bi, ci, di, cond);
 776     } else {
 777         TCGv_vec t = tcg_temp_new_vec(type);
 778         tcg_gen_cmp_vec(cond, vece, t, a, b);
 779         tcg_gen_bitsel_vec(vece, r, t, c, d);
 780         tcg_temp_free_vec(t);
 781     }
 782     tcg_swap_vecop_list(hold_list);
 783 }