tcg/tcg-op-vec.c

   1 /*
   2  * Tiny Code Generator for QEMU
   3  *
   4  * Copyright (c) 2018 Linaro, Inc.
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "qemu/osdep.h"
  21 #include "tcg/tcg.h"
  22 #include "tcg/tcg-op.h"
  23 #include "tcg/tcg-mo.h"
  24 #include "tcg-internal.h"
  25
  26
  27 /* Reduce the number of ifdefs below.  This assumes that all uses of
  28    TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
  29    the compiler can eliminate.  */
  30 #if TCG_TARGET_REG_BITS == 64
  31 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
  32 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
  33 #define TCGV_LOW  TCGV_LOW_link_error
  34 #define TCGV_HIGH TCGV_HIGH_link_error
  35 #endif
  36
  37 /*
  38  * Vector optional opcode tracking.
  39  * Except for the basic logical operations (and, or, xor), and
  40  * data movement (mov, ld, st, dupi), many vector opcodes are
  41  * optional and may not be supported on the host.  Thank Intel
  42  * for the irregularity in their instruction set.
  43  *
  44  * The gvec expanders allow custom vector operations to be composed,
  45  * generally via the .fniv callback in the GVecGen* structures.  At
  46  * the same time, in deciding whether to use this hook we need to
  47  * know if the host supports the required operations.  This is
  48  * presented as an array of opcodes, terminated by 0.  Each opcode
  49  * is assumed to be expanded with the given VECE.
  50  *
  51  * For debugging, we want to validate this array.  Therefore, when
  52  * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders
  53  * will validate that their opcode is present in the list.
  54  */
  55 #ifdef CONFIG_DEBUG_TCG
  56 void tcg_assert_listed_vecop(TCGOpcode op)
  57 {
  58     const TCGOpcode *p = tcg_ctx->vecop_list;
  59     if (p) {
  60         for (; *p; ++p) {
  61             if (*p == op) {
  62                 return;
  63             }
  64         }
  65         g_assert_not_reached();
  66     }
  67 }
  68 #endif
  69
  70 bool tcg_can_emit_vecop_list(const TCGOpcode *list,
  71                              TCGType type, unsigned vece)
  72 {
  73     if (list == NULL) {
  74         return true;
  75     }
  76
  77     for (; *list; ++list) {
  78         TCGOpcode opc = *list;
  79
  80 #ifdef CONFIG_DEBUG_TCG
  81         switch (opc) {
  82         case INDEX_op_and_vec:
  83         case INDEX_op_or_vec:
  84         case INDEX_op_xor_vec:
  85         case INDEX_op_mov_vec:
  86         case INDEX_op_dup_vec:
  87         case INDEX_op_dup2_vec:
  88         case INDEX_op_ld_vec:
  89         case INDEX_op_st_vec:
  90         case INDEX_op_bitsel_vec:
  91             /* These opcodes are mandatory and should not be listed.  */
  92             g_assert_not_reached();
  93         case INDEX_op_not_vec:
  94             /* These opcodes have generic expansions using the above.  */
  95             g_assert_not_reached();
  96         default:
  97             break;
  98         }
  99 #endif
 100
 101         if (tcg_can_emit_vec_op(opc, type, vece)) {
 102             continue;
 103         }
 104
 105         /*
 106          * The opcode list is created by front ends based on what they
 107          * actually invoke.  We must mirror the logic in the routines
 108          * below for generic expansions using other opcodes.
 109          */
 110         switch (opc) {
 111         case INDEX_op_neg_vec:
 112             if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)) {
 113                 continue;
 114             }
 115             break;
 116         case INDEX_op_abs_vec:
 117             if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)
 118                 && (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0
 119                     || tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0
 120                     || tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece))) {
 121                 continue;
 122             }
 123             break;
 124         case INDEX_op_usadd_vec:
 125             if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece) ||
 126                 tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
 127                 continue;
 128             }
 129             break;
 130         case INDEX_op_ussub_vec:
 131             if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece) ||
 132                 tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
 133                 continue;
 134             }
 135             break;
 136         case INDEX_op_cmpsel_vec:
 137         case INDEX_op_smin_vec:
 138         case INDEX_op_smax_vec:
 139         case INDEX_op_umin_vec:
 140         case INDEX_op_umax_vec:
 141             if (tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
 142                 continue;
 143             }
 144             break;
 145         default:
 146             break;
 147         }
 148         return false;
 149     }
 150     return true;
 151 }
 152
 153 void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a)
 154 {
 155     TCGOp *op = tcg_emit_op(opc, 2);
 156     TCGOP_VECL(op) = type - TCG_TYPE_V64;
 157     TCGOP_VECE(op) = vece;
 158     op->args[0] = r;
 159     op->args[1] = a;
 160 }
 161
 162 void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece,
 163                TCGArg r, TCGArg a, TCGArg b)
 164 {
 165     TCGOp *op = tcg_emit_op(opc, 3);
 166     TCGOP_VECL(op) = type - TCG_TYPE_V64;
 167     TCGOP_VECE(op) = vece;
 168     op->args[0] = r;
 169     op->args[1] = a;
 170     op->args[2] = b;
 171 }
 172
 173 void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece,
 174                TCGArg r, TCGArg a, TCGArg b, TCGArg c)
 175 {
 176     TCGOp *op = tcg_emit_op(opc, 4);
 177     TCGOP_VECL(op) = type - TCG_TYPE_V64;
 178     TCGOP_VECE(op) = vece;
 179     op->args[0] = r;
 180     op->args[1] = a;
 181     op->args[2] = b;
 182     op->args[3] = c;
 183 }
 184
 185 static void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r,
 186                       TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e)
 187 {
 188     TCGOp *op = tcg_emit_op(opc, 6);
 189     TCGOP_VECL(op) = type - TCG_TYPE_V64;
 190     TCGOP_VECE(op) = vece;
 191     op->args[0] = r;
 192     op->args[1] = a;
 193     op->args[2] = b;
 194     op->args[3] = c;
 195     op->args[4] = d;
 196     op->args[5] = e;
 197 }
 198
 199 static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a)
 200 {
 201     TCGTemp *rt = tcgv_vec_temp(r);
 202     TCGTemp *at = tcgv_vec_temp(a);
 203     TCGType type = rt->base_type;
 204
 205     /* Must enough inputs for the output.  */
 206     tcg_debug_assert(at->base_type >= type);
 207     vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at));
 208 }
 209
 210 static void vec_gen_op3(TCGOpcode opc, unsigned vece,
 211                         TCGv_vec r, TCGv_vec a, TCGv_vec b)
 212 {
 213     TCGTemp *rt = tcgv_vec_temp(r);
 214     TCGTemp *at = tcgv_vec_temp(a);
 215     TCGTemp *bt = tcgv_vec_temp(b);
 216     TCGType type = rt->base_type;
 217
 218     /* Must enough inputs for the output.  */
 219     tcg_debug_assert(at->base_type >= type);
 220     tcg_debug_assert(bt->base_type >= type);
 221     vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt));
 222 }
 223
 224 void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a)
 225 {
 226     if (r != a) {
 227         vec_gen_op2(INDEX_op_mov_vec, 0, r, a);
 228     }
 229 }
 230
 231 TCGv_vec tcg_const_zeros_vec(TCGType type)
 232 {
 233     TCGv_vec ret = tcg_temp_new_vec(type);
 234     tcg_gen_dupi_vec(MO_64, ret, 0);
 235     return ret;
 236 }
 237
 238 TCGv_vec tcg_const_ones_vec(TCGType type)
 239 {
 240     TCGv_vec ret = tcg_temp_new_vec(type);
 241     tcg_gen_dupi_vec(MO_64, ret, -1);
 242     return ret;
 243 }
 244
 245 TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec m)
 246 {
 247     TCGTemp *t = tcgv_vec_temp(m);
 248     return tcg_const_zeros_vec(t->base_type);
 249 }
 250
 251 TCGv_vec tcg_const_ones_vec_matching(TCGv_vec m)
 252 {
 253     TCGTemp *t = tcgv_vec_temp(m);
 254     return tcg_const_ones_vec(t->base_type);
 255 }
 256
 257 void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a)
 258 {
 259     TCGTemp *rt = tcgv_vec_temp(r);
 260     tcg_gen_mov_vec(r, tcg_constant_vec(rt->base_type, vece, a));
 261 }
 262
 263 void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a)
 264 {
 265     TCGArg ri = tcgv_vec_arg(r);
 266     TCGTemp *rt = arg_temp(ri);
 267     TCGType type = rt->base_type;
 268
 269     if (TCG_TARGET_REG_BITS == 64) {
 270         TCGArg ai = tcgv_i64_arg(a);
 271         vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
 272     } else if (vece == MO_64) {
 273         TCGArg al = tcgv_i32_arg(TCGV_LOW(a));
 274         TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a));
 275         vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah);
 276     } else {
 277         TCGArg ai = tcgv_i32_arg(TCGV_LOW(a));
 278         vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
 279     }
 280 }
 281
 282 void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a)
 283 {
 284     TCGArg ri = tcgv_vec_arg(r);
 285     TCGArg ai = tcgv_i32_arg(a);
 286     TCGTemp *rt = arg_temp(ri);
 287     TCGType type = rt->base_type;
 288
 289     vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
 290 }
 291
 292 void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec r, TCGv_ptr b,
 293                          tcg_target_long ofs)
 294 {
 295     TCGArg ri = tcgv_vec_arg(r);
 296     TCGArg bi = tcgv_ptr_arg(b);
 297     TCGTemp *rt = arg_temp(ri);
 298     TCGType type = rt->base_type;
 299
 300     vec_gen_3(INDEX_op_dupm_vec, type, vece, ri, bi, ofs);
 301 }
 302
 303 static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o)
 304 {
 305     TCGArg ri = tcgv_vec_arg(r);
 306     TCGArg bi = tcgv_ptr_arg(b);
 307     TCGTemp *rt = arg_temp(ri);
 308     TCGType type = rt->base_type;
 309
 310     vec_gen_3(opc, type, 0, ri, bi, o);
 311 }
 312
 313 void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
 314 {
 315     vec_gen_ldst(INDEX_op_ld_vec, r, b, o);
 316 }
 317
 318 void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
 319 {
 320     vec_gen_ldst(INDEX_op_st_vec, r, b, o);
 321 }
 322
 323 void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type)
 324 {
 325     TCGArg ri = tcgv_vec_arg(r);
 326     TCGArg bi = tcgv_ptr_arg(b);
 327     TCGTemp *rt = arg_temp(ri);
 328     TCGType type = rt->base_type;
 329
 330     tcg_debug_assert(low_type >= TCG_TYPE_V64);
 331     tcg_debug_assert(low_type <= type);
 332     vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o);
 333 }
 334
 335 void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 336 {
 337     vec_gen_op3(INDEX_op_and_vec, 0, r, a, b);
 338 }
 339
 340 void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 341 {
 342     vec_gen_op3(INDEX_op_or_vec, 0, r, a, b);
 343 }
 344
 345 void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 346 {
 347     vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b);
 348 }
 349
 350 void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 351 {
 352     if (TCG_TARGET_HAS_andc_vec) {
 353         vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b);
 354     } else {
 355         TCGv_vec t = tcg_temp_new_vec_matching(r);
 356         tcg_gen_not_vec(0, t, b);
 357         tcg_gen_and_vec(0, r, a, t);
 358         tcg_temp_free_vec(t);
 359     }
 360 }
 361
 362 void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 363 {
 364     if (TCG_TARGET_HAS_orc_vec) {
 365         vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b);
 366     } else {
 367         TCGv_vec t = tcg_temp_new_vec_matching(r);
 368         tcg_gen_not_vec(0, t, b);
 369         tcg_gen_or_vec(0, r, a, t);
 370         tcg_temp_free_vec(t);
 371     }
 372 }
 373
 374 void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 375 {
 376     if (TCG_TARGET_HAS_nand_vec) {
 377         vec_gen_op3(INDEX_op_nand_vec, 0, r, a, b);
 378     } else {
 379         tcg_gen_and_vec(0, r, a, b);
 380         tcg_gen_not_vec(0, r, r);
 381     }
 382 }
 383
 384 void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 385 {
 386     if (TCG_TARGET_HAS_nor_vec) {
 387         vec_gen_op3(INDEX_op_nor_vec, 0, r, a, b);
 388     } else {
 389         tcg_gen_or_vec(0, r, a, b);
 390         tcg_gen_not_vec(0, r, r);
 391     }
 392 }
 393
 394 void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 395 {
 396     if (TCG_TARGET_HAS_eqv_vec) {
 397         vec_gen_op3(INDEX_op_eqv_vec, 0, r, a, b);
 398     } else {
 399         tcg_gen_xor_vec(0, r, a, b);
 400         tcg_gen_not_vec(0, r, r);
 401     }
 402 }
 403
 404 static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc)
 405 {
 406     TCGTemp *rt = tcgv_vec_temp(r);
 407     TCGTemp *at = tcgv_vec_temp(a);
 408     TCGArg ri = temp_arg(rt);
 409     TCGArg ai = temp_arg(at);
 410     TCGType type = rt->base_type;
 411     int can;
 412
 413     tcg_debug_assert(at->base_type >= type);
 414     tcg_assert_listed_vecop(opc);
 415     can = tcg_can_emit_vec_op(opc, type, vece);
 416     if (can > 0) {
 417         vec_gen_2(opc, type, vece, ri, ai);
 418     } else if (can < 0) {
 419         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 420         tcg_expand_vec_op(opc, type, vece, ri, ai);
 421         tcg_swap_vecop_list(hold_list);
 422     } else {
 423         return false;
 424     }
 425     return true;
 426 }
 427
 428 void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
 429 {
 430     const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 431
 432     if (!TCG_TARGET_HAS_not_vec || !do_op2(vece, r, a, INDEX_op_not_vec)) {
 433         TCGv_vec t = tcg_const_ones_vec_matching(r);
 434         tcg_gen_xor_vec(0, r, a, t);
 435         tcg_temp_free_vec(t);
 436     }
 437     tcg_swap_vecop_list(hold_list);
 438 }
 439
 440 void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
 441 {
 442     const TCGOpcode *hold_list;
 443
 444     tcg_assert_listed_vecop(INDEX_op_neg_vec);
 445     hold_list = tcg_swap_vecop_list(NULL);
 446
 447     if (!TCG_TARGET_HAS_neg_vec || !do_op2(vece, r, a, INDEX_op_neg_vec)) {
 448         TCGv_vec t = tcg_const_zeros_vec_matching(r);
 449         tcg_gen_sub_vec(vece, r, t, a);
 450         tcg_temp_free_vec(t);
 451     }
 452     tcg_swap_vecop_list(hold_list);
 453 }
 454
 455 void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
 456 {
 457     const TCGOpcode *hold_list;
 458
 459     tcg_assert_listed_vecop(INDEX_op_abs_vec);
 460     hold_list = tcg_swap_vecop_list(NULL);
 461
 462     if (!do_op2(vece, r, a, INDEX_op_abs_vec)) {
 463         TCGType type = tcgv_vec_temp(r)->base_type;
 464         TCGv_vec t = tcg_temp_new_vec(type);
 465
 466         tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece));
 467         if (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0) {
 468             tcg_gen_neg_vec(vece, t, a);
 469             tcg_gen_smax_vec(vece, r, a, t);
 470         } else {
 471             if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) {
 472                 tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1);
 473             } else {
 474                 tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a,
 475                                 tcg_constant_vec(type, vece, 0));
 476             }
 477             tcg_gen_xor_vec(vece, r, a, t);
 478             tcg_gen_sub_vec(vece, r, r, t);
 479         }
 480
 481         tcg_temp_free_vec(t);
 482     }
 483     tcg_swap_vecop_list(hold_list);
 484 }
 485
 486 static void do_shifti(TCGOpcode opc, unsigned vece,
 487                       TCGv_vec r, TCGv_vec a, int64_t i)
 488 {
 489     TCGTemp *rt = tcgv_vec_temp(r);
 490     TCGTemp *at = tcgv_vec_temp(a);
 491     TCGArg ri = temp_arg(rt);
 492     TCGArg ai = temp_arg(at);
 493     TCGType type = rt->base_type;
 494     int can;
 495
 496     tcg_debug_assert(at->base_type == type);
 497     tcg_debug_assert(i >= 0 && i < (8 << vece));
 498     tcg_assert_listed_vecop(opc);
 499
 500     if (i == 0) {
 501         tcg_gen_mov_vec(r, a);
 502         return;
 503     }
 504
 505     can = tcg_can_emit_vec_op(opc, type, vece);
 506     if (can > 0) {
 507         vec_gen_3(opc, type, vece, ri, ai, i);
 508     } else {
 509         /* We leave the choice of expansion via scalar or vector shift
 510            to the target.  Often, but not always, dupi can feed a vector
 511            shift easier than a scalar.  */
 512         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 513         tcg_debug_assert(can < 0);
 514         tcg_expand_vec_op(opc, type, vece, ri, ai, i);
 515         tcg_swap_vecop_list(hold_list);
 516     }
 517 }
 518
 519 void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
 520 {
 521     do_shifti(INDEX_op_shli_vec, vece, r, a, i);
 522 }
 523
 524 void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
 525 {
 526     do_shifti(INDEX_op_shri_vec, vece, r, a, i);
 527 }
 528
 529 void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
 530 {
 531     do_shifti(INDEX_op_sari_vec, vece, r, a, i);
 532 }
 533
 534 void tcg_gen_rotli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
 535 {
 536     do_shifti(INDEX_op_rotli_vec, vece, r, a, i);
 537 }
 538
 539 void tcg_gen_rotri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
 540 {
 541     int bits = 8 << vece;
 542     tcg_debug_assert(i >= 0 && i < bits);
 543     do_shifti(INDEX_op_rotli_vec, vece, r, a, -i & (bits - 1));
 544 }
 545
 546 void tcg_gen_cmp_vec(TCGCond cond, unsigned vece,
 547                      TCGv_vec r, TCGv_vec a, TCGv_vec b)
 548 {
 549     TCGTemp *rt = tcgv_vec_temp(r);
 550     TCGTemp *at = tcgv_vec_temp(a);
 551     TCGTemp *bt = tcgv_vec_temp(b);
 552     TCGArg ri = temp_arg(rt);
 553     TCGArg ai = temp_arg(at);
 554     TCGArg bi = temp_arg(bt);
 555     TCGType type = rt->base_type;
 556     int can;
 557
 558     tcg_debug_assert(at->base_type >= type);
 559     tcg_debug_assert(bt->base_type >= type);
 560     tcg_assert_listed_vecop(INDEX_op_cmp_vec);
 561     can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece);
 562     if (can > 0) {
 563         vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
 564     } else {
 565         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 566         tcg_debug_assert(can < 0);
 567         tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
 568         tcg_swap_vecop_list(hold_list);
 569     }
 570 }
 571
 572 static bool do_op3(unsigned vece, TCGv_vec r, TCGv_vec a,
 573                    TCGv_vec b, TCGOpcode opc)
 574 {
 575     TCGTemp *rt = tcgv_vec_temp(r);
 576     TCGTemp *at = tcgv_vec_temp(a);
 577     TCGTemp *bt = tcgv_vec_temp(b);
 578     TCGArg ri = temp_arg(rt);
 579     TCGArg ai = temp_arg(at);
 580     TCGArg bi = temp_arg(bt);
 581     TCGType type = rt->base_type;
 582     int can;
 583
 584     tcg_debug_assert(at->base_type >= type);
 585     tcg_debug_assert(bt->base_type >= type);
 586     tcg_assert_listed_vecop(opc);
 587     can = tcg_can_emit_vec_op(opc, type, vece);
 588     if (can > 0) {
 589         vec_gen_3(opc, type, vece, ri, ai, bi);
 590     } else if (can < 0) {
 591         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 592         tcg_expand_vec_op(opc, type, vece, ri, ai, bi);
 593         tcg_swap_vecop_list(hold_list);
 594     } else {
 595         return false;
 596     }
 597     return true;
 598 }
 599
 600 static void do_op3_nofail(unsigned vece, TCGv_vec r, TCGv_vec a,
 601                           TCGv_vec b, TCGOpcode opc)
 602 {
 603     bool ok = do_op3(vece, r, a, b, opc);
 604     tcg_debug_assert(ok);
 605 }
 606
 607 void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 608 {
 609     do_op3_nofail(vece, r, a, b, INDEX_op_add_vec);
 610 }
 611
 612 void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 613 {
 614     do_op3_nofail(vece, r, a, b, INDEX_op_sub_vec);
 615 }
 616
 617 void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 618 {
 619     do_op3_nofail(vece, r, a, b, INDEX_op_mul_vec);
 620 }
 621
 622 void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 623 {
 624     do_op3_nofail(vece, r, a, b, INDEX_op_ssadd_vec);
 625 }
 626
 627 void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 628 {
 629     if (!do_op3(vece, r, a, b, INDEX_op_usadd_vec)) {
 630         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 631         TCGv_vec t = tcg_temp_new_vec_matching(r);
 632
 633         /* usadd(a, b) = min(a, ~b) + b */
 634         tcg_gen_not_vec(vece, t, b);
 635         tcg_gen_umin_vec(vece, t, t, a);
 636         tcg_gen_add_vec(vece, r, t, b);
 637
 638         tcg_temp_free_vec(t);
 639         tcg_swap_vecop_list(hold_list);
 640     }
 641 }
 642
 643 void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 644 {
 645     do_op3_nofail(vece, r, a, b, INDEX_op_sssub_vec);
 646 }
 647
 648 void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 649 {
 650     if (!do_op3(vece, r, a, b, INDEX_op_ussub_vec)) {
 651         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 652         TCGv_vec t = tcg_temp_new_vec_matching(r);
 653
 654         /* ussub(a, b) = max(a, b) - b */
 655         tcg_gen_umax_vec(vece, t, a, b);
 656         tcg_gen_sub_vec(vece, r, t, b);
 657
 658         tcg_temp_free_vec(t);
 659         tcg_swap_vecop_list(hold_list);
 660     }
 661 }
 662
 663 static void do_minmax(unsigned vece, TCGv_vec r, TCGv_vec a,
 664                       TCGv_vec b, TCGOpcode opc, TCGCond cond)
 665 {
 666     if (!do_op3(vece, r, a, b, opc)) {
 667         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 668         tcg_gen_cmpsel_vec(cond, vece, r, a, b, a, b);
 669         tcg_swap_vecop_list(hold_list);
 670     }
 671 }
 672
 673 void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 674 {
 675     do_minmax(vece, r, a, b, INDEX_op_smin_vec, TCG_COND_LT);
 676 }
 677
 678 void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 679 {
 680     do_minmax(vece, r, a, b, INDEX_op_umin_vec, TCG_COND_LTU);
 681 }
 682
 683 void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 684 {
 685     do_minmax(vece, r, a, b, INDEX_op_smax_vec, TCG_COND_GT);
 686 }
 687
 688 void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 689 {
 690     do_minmax(vece, r, a, b, INDEX_op_umax_vec, TCG_COND_GTU);
 691 }
 692
 693 void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 694 {
 695     do_op3_nofail(vece, r, a, b, INDEX_op_shlv_vec);
 696 }
 697
 698 void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 699 {
 700     do_op3_nofail(vece, r, a, b, INDEX_op_shrv_vec);
 701 }
 702
 703 void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 704 {
 705     do_op3_nofail(vece, r, a, b, INDEX_op_sarv_vec);
 706 }
 707
 708 void tcg_gen_rotlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 709 {
 710     do_op3_nofail(vece, r, a, b, INDEX_op_rotlv_vec);
 711 }
 712
 713 void tcg_gen_rotrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 714 {
 715     do_op3_nofail(vece, r, a, b, INDEX_op_rotrv_vec);
 716 }
 717
 718 static void do_shifts(unsigned vece, TCGv_vec r, TCGv_vec a,
 719                       TCGv_i32 s, TCGOpcode opc)
 720 {
 721     TCGTemp *rt = tcgv_vec_temp(r);
 722     TCGTemp *at = tcgv_vec_temp(a);
 723     TCGTemp *st = tcgv_i32_temp(s);
 724     TCGArg ri = temp_arg(rt);
 725     TCGArg ai = temp_arg(at);
 726     TCGArg si = temp_arg(st);
 727     TCGType type = rt->base_type;
 728     int can;
 729
 730     tcg_debug_assert(at->base_type >= type);
 731     tcg_assert_listed_vecop(opc);
 732     can = tcg_can_emit_vec_op(opc, type, vece);
 733     if (can > 0) {
 734         vec_gen_3(opc, type, vece, ri, ai, si);
 735     } else if (can < 0) {
 736         const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
 737         tcg_expand_vec_op(opc, type, vece, ri, ai, si);
 738         tcg_swap_vecop_list(hold_list);
 739     } else {
 740         g_assert_not_reached();
 741     }
 742 }
 743
 744 void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
 745 {
 746     do_shifts(vece, r, a, b, INDEX_op_shls_vec);
 747 }
 748
 749 void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
 750 {
 751     do_shifts(vece, r, a, b, INDEX_op_shrs_vec);
 752 }
 753
 754 void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
 755 {
 756     do_shifts(vece, r, a, b, INDEX_op_sars_vec);
 757 }
 758
 759 void tcg_gen_rotls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s)
 760 {
 761     do_shifts(vece, r, a, s, INDEX_op_rotls_vec);
 762 }
 763
 764 void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a,
 765                         TCGv_vec b, TCGv_vec c)
 766 {
 767     TCGTemp *rt = tcgv_vec_temp(r);
 768     TCGTemp *at = tcgv_vec_temp(a);
 769     TCGTemp *bt = tcgv_vec_temp(b);
 770     TCGTemp *ct = tcgv_vec_temp(c);
 771     TCGType type = rt->base_type;
 772
 773     tcg_debug_assert(at->base_type >= type);
 774     tcg_debug_assert(bt->base_type >= type);
 775     tcg_debug_assert(ct->base_type >= type);
 776
 777     if (TCG_TARGET_HAS_bitsel_vec) {
 778         vec_gen_4(INDEX_op_bitsel_vec, type, MO_8,
 779                   temp_arg(rt), temp_arg(at), temp_arg(bt), temp_arg(ct));
 780     } else {
 781         TCGv_vec t = tcg_temp_new_vec(type);
 782         tcg_gen_and_vec(MO_8, t, a, b);
 783         tcg_gen_andc_vec(MO_8, r, c, a);
 784         tcg_gen_or_vec(MO_8, r, r, t);
 785         tcg_temp_free_vec(t);
 786     }
 787 }
 788
 789 void tcg_gen_cmpsel_vec(TCGCond cond, unsigned vece, TCGv_vec r,
 790                         TCGv_vec a, TCGv_vec b, TCGv_vec c, TCGv_vec d)
 791 {
 792     TCGTemp *rt = tcgv_vec_temp(r);
 793     TCGTemp *at = tcgv_vec_temp(a);
 794     TCGTemp *bt = tcgv_vec_temp(b);
 795     TCGTemp *ct = tcgv_vec_temp(c);
 796     TCGTemp *dt = tcgv_vec_temp(d);
 797     TCGArg ri = temp_arg(rt);
 798     TCGArg ai = temp_arg(at);
 799     TCGArg bi = temp_arg(bt);
 800     TCGArg ci = temp_arg(ct);
 801     TCGArg di = temp_arg(dt);
 802     TCGType type = rt->base_type;
 803     const TCGOpcode *hold_list;
 804     int can;
 805
 806     tcg_debug_assert(at->base_type >= type);
 807     tcg_debug_assert(bt->base_type >= type);
 808     tcg_debug_assert(ct->base_type >= type);
 809     tcg_debug_assert(dt->base_type >= type);
 810
 811     tcg_assert_listed_vecop(INDEX_op_cmpsel_vec);
 812     hold_list = tcg_swap_vecop_list(NULL);
 813     can = tcg_can_emit_vec_op(INDEX_op_cmpsel_vec, type, vece);
 814
 815     if (can > 0) {
 816         vec_gen_6(INDEX_op_cmpsel_vec, type, vece, ri, ai, bi, ci, di, cond);
 817     } else if (can < 0) {
 818         tcg_expand_vec_op(INDEX_op_cmpsel_vec, type, vece,
 819                           ri, ai, bi, ci, di, cond);
 820     } else {
 821         TCGv_vec t = tcg_temp_new_vec(type);
 822         tcg_gen_cmp_vec(cond, vece, t, a, b);
 823         tcg_gen_bitsel_vec(vece, r, t, c, d);
 824         tcg_temp_free_vec(t);
 825     }
 826     tcg_swap_vecop_list(hold_list);
 827 }